package com.changxianggu.crawler.cloud.test;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import com.changxianggu.cloud.crawler.core.WebApplication;
import com.changxianggu.cloud.crawler.core.modules.storage.entity.CxgBook;
import com.changxianggu.cloud.crawler.core.modules.storage.service.ICxgBookService;
import com.changxianggu.cloud.crawler.core.utils.HttpUtil;
import com.changxianggu.cloud.crawler.core.utils.UserAgentUtil;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = {WebApplication.class})// 指定启动类
@Slf4j
public class DainZiSheYueDu {

    @Autowired
    ICxgBookService bookService;

    public static final String COOKIE = "rsaKeys=%7B%22privateKey%22%3A%22-----BEGIN%20RSA%20PRIVATE%20KEY-----%5Cr%5CnMIIEoQIBAAKCAQEAkVbFfxndGS26cjnBJc%2F1V%2FKlnQoTZHotXbdMe%2FfkFfpazxPn%5Cr%5CnF3RqVuEmCCTP1uZZlQ2NA6CGFVkV%2B%2FhB8qxmo4OVj99juOMJgzcptYCdSLHeFhKO%5Cr%5CnbI4h5copNBn7d7KRJK%2FTZkwToNK9MO4Gm2MtLuNCWcVk47Rdtr4mtuss8ml%2BVsvK%5Cr%5CnpqYd5DhqP6Q0cMdwcHCdy9Gt4WLTGmTcQtTI1JOblJ3IRkWA9A6usk17LXntD31A%5Cr%5CnB89pEby623RLSR6eB8Kvexhju7LNZbjbT0Dwg2Xihtoo2phEA6USN6WFf5xwMB%2Fv%5Cr%5Cnsg%2Fwb8x9UvAQjWqcCRFJhTYe493gpG3%2B2Da2fQIBAwKCAQBg5IOqET4QySb20Stu%5Cr%5CniqOP9xkTXAztpsjpJN2n%2BpgOpuc0t%2B9k%2BEbklhlawzU57uZjXl4CawQOO2P9UCv3%5Cr%5CnHZnCV7kKlO0l7LECJMZ5AGjbIT65YbRIXsFD3Bt4EVJPzGDDH%2BJEMrfAjH4gnq8S%5Cr%5CnQh4fQiw72O3tIukkfsR58h327%2FtSy%2B6ZD8sagWfvE6WQMaL0LXerBOlJoEBbSEBN%5Cr%5CnGD1VdUL92T0wdJ5%2FI0GL5eeJKmeaSd5NpcwPucqPbuG67yTA3uFcX9tWxAKKZGk%2B%5Cr%5CnmcsokO4%2F3jCp47CH1WfC611osxiClT2040RYMfaWymGQIK0cghCJVjItsa%2B5yRhD%5Cr%5CnN6%2BTAoGBANz9Sjxkx7RV7esvkkn%2Fx0etBmT9wgqUiwHDu0s%2FOyz6jjntP2yzhdy%2B%5Cr%5CnL%2B477tImOKdcz56pfY8h6GxvDx9LnFmhNvrnUPfXXEvzrnDNRK6N7%2BPgwCfwKCQ%2F%5Cr%5CnxqZkyFDUrv7QOe2C14Okh21QbL6D0KvPy%2Fhp09pLbprrWhG5iMRrAoGBAKhdT6hc%5Cr%5CnRrjmiGMpdHIYtbRVJgVNiEVeZ3DNvTCP9T%2FZ1X0qmH%2FKELZ3Kd4khc25gapLPtPp%5Cr%5Cn2SfYPZ6VPtllG41FPWKJN6LiMH2ic8ykFZE4nJlGTLfrmpb4HlAufCiN4MycBiWf%5Cr%5Cn44ZG%2Bh4gU7%2BxaOyOgjhNYPlT5L4iZki52mq3AoGBAJNThtLt2ng5SUd1DDFVL4Ue%5Cr%5CnBENT1rG4XKvX0jIqJ3NRtCaeKkh3rpMpdUl9SeFu0G%2BTNRRw%2Fl9r8EhKChTdEuZr%5Cr%5CnefyaNfqPkt1NHvXeLcmz9UKV1W%2F1cBgqhG7t2uCNyf81e%2FOsj60YWkjgSH8Cix01%5Cr%5CnMqWb4pGHnxHyPAvRBdhHAoGAcD41GugvJe8Fl3D4TBB5IuNurjOwLj7voIkoywqj%5Cr%5Cnf%2BaOU3G6%2F9wLJE9xPsMD3nurxtzUjUaQxTrTvw4p5kNns4N%2BQbDPwewgU8GiiG1j%5Cr%5CntiW9u4QzJUe8ZKVpismoGwlAiGgEGRVCWYSmvsA31SDwnbRW0DOV%2B41DKWxEMHvm%5Cr%5Cn8c8CgYAmFnnYPXVIHsKB5Y4dPWJ1DGye1DpdWK0rTLFd5yvmz0wPhEH9YL07oxjH%5Cr%5CnIGg0KnIlXuC8l0pajUSEEmY1w6D86BL8zRwXUI3myQO5vDF%2BeddGcP9MEiv4jIWu%5Cr%5CnyFOL8%2Bjj8Z6AnqWCLC2bvNljzv8AdcXfYHZUVwWDsBRUE1OCUQ%3D%3D%5Cr%5Cn-----END%20RSA%20PRIVATE%20KEY-----%5Cr%5Cn%22%2C%22publicKey%22%3A%22-----BEGIN%20PUBLIC%20KEY-----%5Cr%5CnMIIBIDANBgkqhkiG9w0BAQEFAAOCAQ0AMIIBCAKCAQEAkVbFfxndGS26cjnBJc%2F1%5Cr%5CnV%2FKlnQoTZHotXbdMe%2FfkFfpazxPnF3RqVuEmCCTP1uZZlQ2NA6CGFVkV%2B%2FhB8qxm%5Cr%5Cno4OVj99juOMJgzcptYCdSLHeFhKObI4h5copNBn7d7KRJK%2FTZkwToNK9MO4Gm2Mt%5Cr%5CnLuNCWcVk47Rdtr4mtuss8ml%2BVsvKpqYd5DhqP6Q0cMdwcHCdy9Gt4WLTGmTcQtTI%5Cr%5Cn1JOblJ3IRkWA9A6usk17LXntD31AB89pEby623RLSR6eB8Kvexhju7LNZbjbT0Dw%5Cr%5Cng2Xihtoo2phEA6USN6WFf5xwMB%2Fvsg%2Fwb8x9UvAQjWqcCRFJhTYe493gpG3%2B2Da2%5Cr%5CnfQIBAw%3D%3D%5Cr%5Cn-----END%20PUBLIC%20KEY-----%5Cr%5Cn%22%7D; JSESSIONID=A51CDFE5DF19D524F62E984CA2CA6C10; Hm_lvt_85b41e32febbcaf90f263ad5f4c6cd6f=1582264950,1582265479,1582265496,1582512718; Hm_lpvt_85b41e32febbcaf90f263ad5f4c6cd6f=1582512776";
    public static final String ORIGIN = "https://yd.51zhy.cn";
    public static final String REFERER = "https://yd.51zhy.cn/ebook/web/search/search4HomePage?keywords=&keywords4srch=&isbn4srch=&clc14srch=&clc24srch=&clc34srch=&startDate4srch=&endDate4srch=&queryType=single&formReq=yes&classGrade=&class4srch=&clc4srch=&keywords=&tst=m5&orderColumn=score&orderType=DESC&currentPage=1&countsOfPerPage=15";
    public static final String URL_PRE = "https://yd.51zhy.cn/ebook/web/newBook/queryNewBookById?id=";


    @Test
    public void initCrawler() throws Exception {
        doCrawler();
    }

    public void doCrawler() throws Exception {
        int pageNum = 1349;
        int totalPage = 1371;
        for (int i = pageNum; i <= totalPage; i++) {
            String url = "https://yd.51zhy.cn/ebook/web/search/search4HomePage?queryType=single&formReq=yes&tst=m5&orderColumn=score&orderType=DESC&currentPage={}&countsOfPerPage=15";
            String formatUrl = StrUtil.format(url, i);
            Map<String, Object> headers = MapUtil.newHashMap(5);
            headers.put("Cookie", COOKIE);
            headers.put("User-Agent", UserAgentUtil.random());
            headers.put("Origin", ORIGIN);
            headers.put("Referer", REFERER);
            String result = HttpUtil.doGet(formatUrl, headers, MapUtil.newHashMap());
            int tryCount = 1;
            while (result.contains("页面没有找到") && tryCount <= 5) {
                //睡眠2秒继续访问
                Thread.sleep(2000);
                result = HttpUtil.doPost(url, headers);
                tryCount++;
            }
            if (StrUtil.isNotBlank(result)) {
                Document htmlDoc = null;
                try {
                    htmlDoc = Jsoup.parse(result);
                } catch (Exception e) {
                    log.warn("jsoup解析html发生错误");
                }
                if (htmlDoc == null) {
                    continue;
                }
                Elements bookContentList = htmlDoc.select(".search-list-box .ebook-item");
                if (bookContentList != null && bookContentList.size() > 0) {
                    List<CxgBook> listBook = new ArrayList<>();
                    for (Element element : bookContentList) {
                        CxgBook bookEntity = new CxgBook();
                        Element pkEle = element.select(".search_Input").first();
                        String pkId = "";
                        if (pkEle != null) {
                            pkId = pkEle.attr("value");
                            bookEntity.setPkId(pkId);
                            bookEntity.setDigitalUri(URL_PRE + pkId);
                        }
                        Element imgEle = element.select(".book img").first();
                        if (imgEle != null) {
                            bookEntity.setCover(imgEle.attr("src"));
                        }
                        Element titleEle = element.select(".info .wrap a").first();
                        if (titleEle != null) {
                            bookEntity.setBookName(titleEle.attr("title"));
                        }
                        Element authorEle = element.select(".info .wrap .author span").first();
                        if (authorEle != null) {
                            bookEntity.setAuthor(authorEle.text());
                        }
                        Element dateEle = element.select(".info .wrap .date").first();
                        if (dateEle != null) {
                            String text = dateEle.text();
                            String[] strings = text.split("：");
                            if (strings.length == 2) {
                                String pubDateStr = strings[1] + " 00:00:00";
                                DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
                                LocalDateTime localDateTime = LocalDateTime.parse(pubDateStr, dtf);
                                bookEntity.setPubDate(localDateTime);
                            }
                        }
                        Element isbnEle = element.select(".info .isbn").first();
                        if (isbnEle != null) {
                            String[] strings = isbnEle.text().split("：");
                            if (strings.length == 2) {
                                String isbn = strings[1].replaceAll("-", "").replaceAll("—", "");
                                bookEntity.setBookIsbn(isbn);
                            }
                        }

                        CxgBook tmpBookEntity = getBookEntity(pkId);
                        if (tmpBookEntity != null) {
                            bookEntity.setPageNum(tmpBookEntity.getPageNum());
                            bookEntity.setBookVersion(tmpBookEntity.getBookVersion());
                            bookEntity.setBrief(tmpBookEntity.getBrief());
                            bookEntity.setCatalog(tmpBookEntity.getCatalog());
                            bookEntity.setPrice(tmpBookEntity.getPrice());
                        }
                        bookEntity.setIsSync(0);
                        bookEntity.setCreateTime(LocalDateTime.now());
                        bookEntity.setPressName("电子工业出版社");
                        bookEntity.setStatus(1);
                        bookEntity.setPressId(35);
                        log.info(bookEntity.getBookName());
                        listBook.add(bookEntity);
                    }
                    bookService.saveBatch(listBook);
                    log.info("第" + i + "页数据保存成功");
                    log.warn("电子社--->" + i);
                }
            }
        }
    }

    public CxgBook getBookEntity(String numId) throws Exception {
        CxgBook bookEntity = null;
        if (StrUtil.isBlank(numId)) {
            return bookEntity;
        }
        String url = "https://yd.51zhy.cn/ebook/web/newBook/queryNewBookById?id={}";
        Map<String, Object> headers = MapUtil.newHashMap(5);
        headers.put("Cookie", COOKIE);
        headers.put("User-Agent", UserAgentUtil.random());
        headers.put("Origin", ORIGIN);
        headers.put("Referer", REFERER);
        String formatUrl = StrUtil.format(url, numId);
        String result = HttpUtil.doGet(formatUrl, headers, MapUtil.newHashMap());
        int tryCount = 1;
        while (result.contains("页面没有找到") && tryCount <= 5) {
            //睡眠2秒继续访问
            Thread.sleep(2000);
            result = HttpUtil.doGet(formatUrl, headers);
            tryCount++;
        }
        if (StrUtil.isNotBlank(result)) {
            bookEntity = new CxgBook();
            Document htmlDoc = Jsoup.parse(result);
            Element briefEle = htmlDoc.select("#1 .txt").first();
            Element infoEle = htmlDoc.select(".book-info .txt").first();
            if (infoEle != null) {
                String[] strings = infoEle.html().split("<br>");
                for (String s : strings) {
                    String trimStr = s.trim();
                    if (trimStr.startsWith("页数：")) {
                        String[] strings1 = trimStr.split("：");
                        if (strings1.length == 2) {
                            bookEntity.setPageNum(Integer.parseInt(getNumbers(strings1[1])));
                        }
                    }
                    if (trimStr.startsWith("<span")) {
                        Document document = Jsoup.parseBodyFragment(trimStr);
                        Element element = document.select("s").first();
                        String priceStr = getNumbers(element.text());
                        if (StrUtil.isNotBlank(priceStr)) {
                            bookEntity.setPrice(BigDecimal.valueOf(Double.parseDouble(priceStr)));
                        }
                    }

                }
            }
            bookEntity.setCatalog(getCatalog(numId));
            if (briefEle != null) {
                bookEntity.setBrief(briefEle.text());
            }

        }
        return bookEntity;
    }

    public String getCatalog(String numId) throws Exception {
        String catalog = "";
        String url = "https://yd.51zhy.cn/ebook/web/newBook/getBoolMarksList";
        Map<String, Object> headers = MapUtil.newHashMap(5);
        headers.put("Cookie", COOKIE);
        headers.put("User-Agent", UserAgentUtil.random());
        headers.put("Origin", ORIGIN);
        headers.put("Referer", REFERER);
        String formatUrl = StrUtil.format(url, numId);
        Map<String, Object> paramMap = MapUtil.newHashMap(2);
        paramMap.put("id", numId);
        String result = HttpUtil.doPost(formatUrl, headers, paramMap);
        int tryCount = 1;
        while (result.contains("页面没有找到") && tryCount <= 5) {
            //睡眠2秒继续访问
            Thread.sleep(2000);
            result = HttpUtil.doGet(formatUrl, headers);
            tryCount++;
        }
        if (StrUtil.isNotBlank(result)) {
            Document htmlDoc = Jsoup.parse(result);
            Element catalogEle = htmlDoc.getElementById("showAll");
            if (catalogEle != null) {
                catalog = catalogEle.text();
            }
        }
        return catalog;
    }

    /**
     * 判断一个字符串是否都为数字
     *
     * @param strNum
     * @return
     */
    public boolean isDigit(String strNum) {
        Pattern pattern = Pattern.compile("[0-9]{1,}");
        Matcher matcher = pattern.matcher((CharSequence) strNum);
        return matcher.matches();
    }

    /**
     * 截取数字
     *
     * @param content
     * @return
     */
    public String getNumbers(String content) {
        Pattern pattern = Pattern.compile("\\d+");
        Matcher matcher = pattern.matcher(content);
        while (matcher.find()) {
            return matcher.group(0);
        }
        return "";
    }
}
